from dtree import *
from dtreeID3 import *
import time
import learning as l
import math
from Categories import Categories
from Words import Words
from Files import Files
from CatWordsMatrix import CatWordsMatrix
from FileWordsMatrix import FileWordsMatrix

print "[", time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), "] START"

"Instances de Words"
allwords = Words()

"Instance de Categories"
categories = Categories()

"Instance de Files"
files = Files()

print "[", time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), "] Read StopWords START"
"Ensemble des stopwords"
stopwords = l.readStopWords()
print "[", time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), "] Read StopWords STOP"

print "[", time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), "] Read Learn Files START"
"Nombre des fichiers lus"
folders = ('awards_1990','awards_1991','awards_1992')
l.readFiles(l.PATH, folders, allwords, categories, stopwords, files)
print "[", time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), "] Read Learn Files STOP"

print "[", time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), "] Load Matrix2 START"
"Chargement de la matrice Categories x Words"
#matrix2 = CatWordsMatrix()
#matrix2.loadMatrix(categories,allwords,files)
print "[", time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), "] Load Matrix2 STOP"

print "[", time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), "] Prune matrixFreq START"
"Prune matrixFreq"
#matrix2.pruneMatrix(l.MIN_VAR)
#matrix2.save()
print "[", time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), "] Prune matrixFreq STOP"
print "[", time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), "] STOP"

print "[", time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), "] Load Matrix3 START"
"Chargement de la matrice Files x Words"
matrix3 = FileWordsMatrix()
matrix3.loadMatrix(categories,allwords,files)
print "[", time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), "] Load Matrix3 STOP"

print "[", time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), "] Prune matrix3Freq START"
"Prune matrixFreq"
#matrix3.pruneMatrix(l.MIN_VAR)
#matrix3.save()
print "[", time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), "] Prune matrix3Freq STOP"
print "[", time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), "] STOP"

nbFiles = len(files.bag)
nbCats = len(categories.bag)
nbWords = len(allwords.bag)
'''
mFq = matrix2.matrixFreq
mNb = matrix2.matrixNbFiles
mRw = matrix2.retraceWords
mC = matrix2.categories
mW = matrix2.words

def p(i):
    return mC[i][1]*1.0/nbFiles
def log2(i):
    return math.log(i) / math.log(2)
def h(i):
    x = p(i)
    if x == 0:
        return 0
    return x*log2(x)
def e():
    return -1.0 * sum(map(lambda x: h(x[0]), mC.items()))

def BestCat(word):
	return mC[mFq[:,mRw[allwords.bag[word][0]]].toarray().argmax()]

def CatList(f):
    fw = f[1][1]
    fc = f[1][2]
    fwi = fw.items()
    fwi.sort(key=lambda x: x[1], reverse=True)
    fwi = filter( lambda x: allwords.bag.keys().__contains__(x[0]), fwi)
    fwi = filter( lambda x: mRw.keys().__contains__(allwords.bag[x[0]][0]), fwi)
    fwc = map( lambda x: (x[0],x[1],BestCat(x[0])), fwi)
    for t in fwc:
        if t[2][0] == fc:
            print '*',
        print fc, ' => ', t
	
def cl(i):
	CatList(files.bag.items()[i])

for i in range(0,10,1):
    print "cl(",i,")"
    cl(i)
'''
print "[", time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), "] Building the Data !!!"
target_attr = "THEFinalClassChoosen"
matrix3.words[matrix3.words.__len__()] = [target_attr,0]
attributes = [attr for attr in map(lambda x: x[1][0],matrix3.words.items())]
data = []
for i in range(matrix3.matrixFreq.shape[0]):
    dic = {}
    for j in range(matrix3.matrixFreq.shape[1]):
        dic[attributes[j]] = matrix3.matrixFreq[i,j]
    data.append(dic)
    #data.append(dict(zip(attributes,[datum for datum in matrix3.matrixFreq[i,:]])))
print "[", time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), "] Building the Tree !!!"
tree = create_decision_tree(data, attributes, target_attr, gain)

print "[", time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), "] Tree Build !!"
